# -*- coding：utf-8 -*-
import scrapy
import re
import datetime
from scrapy.http import Request
from urllib import parse
from scrapy.loader import ItemLoader

from ArticleSpider.items import ArticleItem, ArticlItemLoader

from ArticleSpider.utils.common import get_md5
from selenium import webdriver
from pydispatch import dispatcher
from scrapy import signals


class SegmentfaultSpider(scrapy.Spider):
    name = 'sifou'
    allowed_domains = ['segmentfault.com']
    start_urls = ['https://segmentfault.com']

    # def __init__(self):
    #     self.browser = webdriver.Chrome(executable_path="G:/workspace/chromedriver.exe")
    #     super(SegmentfaultSpider, self).__init__()
    #     dispatcher.connect(self.spider_closed, signals.spider_closed)
    #
    # def spider_closed(self, spider):
    #     #当爬虫退出的时候关闭Chrome
    #     print("spider closed")
    #     self.browser.quit()

    def parse(self, response):
        post_notes = response.css(".news-list .news__item-info")
        for post_note in post_notes:
            fav_nums = post_note.css(".votes-num::text").extract()
            post_note = post_note.css("a")
            image_url = post_note.css("::attr(style)").extract_first("")
            image_url = re.findall(r'[(](.*?)[)]', image_url)
            post_url = post_note.css("::attr(href)").extract_first("")
            yield Request(url=parse.urljoin(response.url, post_url),
                          meta={"front_image_url": image_url, "fav_nums": fav_nums}, callback=self.parse_detail)

    @staticmethod
    def parse_detail(response):
        # 通过ItemLoader加载item
        front_image_url = response.meta.get("front_image_url", "")  # 文章背景图
        item_loader = ArticlItemLoader(item=ArticleItem(), response=response)
        item_loader.add_css('title', ".h2 a::text")
        item_loader.add_value("url", response.url)
        item_loader.add_css("create_date", ".text-secondary::text")
        item_loader.add_css("article_text", ".article")
        # item_loader.add_value("article_text","s")
        fav_nums = response.meta.get("fav_nums", 0)
        item_loader.add_value("fav_nums", fav_nums)
        item_loader.add_css("read_volume", ".text-secondary .col-6::text")
        item_loader.add_value("url_object_id", get_md5(response.url))
        item_loader.add_value("front_image_url", front_image_url)

        article_item = item_loader.load_item()

        yield article_item
